from pymongo import MongoClient
from bs4 import BeautifulSoup
from pprint import pprint
import requests
import datetime
import json
import time
import re

server_client = MongoClient('localhost', 27017)

server_db = server_client['knx_posts_db']
offical_posts_coll = server_db['offical_posts_coll']


class POST():
    def __init__(self):
        self.scrapy()

    def scrapy(self):
        url = 'http://crc.wintalent.cn/wt/CRC/web/templet1000/index/corpwebPosition1000CRC!getPostListByCondition?pc.rowSize=10000000&pc.currentPage=1&recruitType=1&releaseTime=0&trademark=0&brandCode=1&showComp=true&searchSuffix=&keyWord=&comPart=0&comPartStr=&crcOrgId=&workPlaceCode=&workPlaceNameV=0&comPartV=0&keyWordV=&positionNameV='
        r = requests.get(url)
        r.encoding = 'utf-8'
        soup = BeautifulSoup(r.text)

        for jd in soup.select('.job_listings .single'):
            url = 'http://crc.wintalent.cn' + jd.find('a').get('href')
            r = requests.get(url)
            r.encoding = 'utf-8'
            detail_soup = BeautifulSoup(r.text)

            name = detail_soup.find(class_ = 'c_orange').get_text()
            location = detail_soup.find(text = re.compile('工作地点：')).parent.get_text().split('：')[1]

            try:
                count = int(re.match('\d+', detail_soup.find(text = re.compile('招聘人数：')).parent.get_text().split('：')[1]).group())
            except:
                count = -1

            date = jd.find('h5').select('span')[-1].text

            description = detail_soup.find(text = re.compile('岗位职责：')).parent.find_next_sibling().get_text() + detail_soup.find(text = re.compile('任职资格：')).parent.find_next_sibling().get_text()

            item = {
                "url": url,
                'edu': '',
                'exp': [],
                'name': name,
                'date': date,
                'lang': '',
                'place': '',
                'major': '',
                'count': count,
                'salary': [],
                'toSchool': True,
                'welfare': [],
                'funType': '',
                'company': '华润集团',
                'location': location,
                'industry': '',
                'keywords': [],
                'platform': 'offical',
                'searchKeyword': '',
                'description': description.strip(),
                'subIndustry': '',
                'stime': datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
            }

            result = offical_posts_coll.replace_one({'company': '华润集团', 'name': name, 'location': location}, item, True)

            if result.matched_count:
                print('-' * 40, 'update one job', '-' * 40)
            else:
                print('-' * 40, 'insert one job', '-' * 40)

            pprint(item)


POST()
